package thesis;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.StringTokenizer;

import com.cybozu.labs.langdetect.Detector;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.cybozu.labs.langdetect.Language;
import com.mongodb.DBObject;


public class FilteringAlgorithm {
	private static final String PROFILEDIR = "lib\\profiles";
	private static final double PROBTHRESHOLD = 0.7;
	
	private HashSet<String> filteringWords = new HashSet<String>();
	private HashSet<String> filteringFunWords = new HashSet<String>();
	private HashSet<String> stopWords = new HashSet<String>();
	
	private LocationManager lMng;
	
	public FilteringAlgorithm(LocationManager lMng) {
		this.lMng = lMng;
		
		getFilteringWords();
		getStopWords();
		
		try {
			DetectorFactory.loadProfile(PROFILEDIR);
		} catch (LangDetectException e) {
			e.printStackTrace();
			System.exit(-1);
		}
	}
	
	public boolean mustBeFiltered(DBObject tweet){
		
		InputTweetInfoExtractor infoExtr = new InputTweetInfoExtractor(tweet);
		
		
		//FILTERING BASED ON GEO COORDINATES
		if(!lMng.isInsideAreaOfEvent(tweet)){
			return true;	
		}
		
		
		//FILTERING BASED ON TEXT
		String text = infoExtr.getText();
		String temp = text.toLowerCase();
		
		if(temp.startsWith("rt")){
			return true;
		}
		
		for (int i=0; i<temp.length(); i++){
			char k = temp.charAt(i);
			if(k != ':' && k != '@' && k != ')' && k != '(' && k != ';' && k != '-' && k != '_')
				if (Character.getNumericValue(k) == -1){
					temp = temp.substring(0, i) + " " + temp.substring(i+1, temp.length());
				}
		}

		StringTokenizer tk1 = new StringTokenizer(temp);
		
		while (tk1.hasMoreTokens()){
			String word = tk1.nextToken();
			
			if(filteringFunWords.contains(word)){
				return true;
			}
		}
		
		temp = temp.replace(':', ' ');
		temp = temp.replace(';', ' ');
		temp = temp.replace('(', ' ');
		temp = temp.replace(')', ' ');
				
		StringTokenizer tk2 = new StringTokenizer(temp);
		while (tk2.hasMoreTokens()){
			String word = tk2.nextToken();
				
			if(filteringWords.contains(word)){
				return true;
			}
		}
		
		
		//FILTERING BASED ON ENGLISH
		Detector detector = null;
		try {
			detector = DetectorFactory.create();
		} catch (LangDetectException e) {
			e.printStackTrace();
			System.exit(-1);
		}
		detector.append(text);
		ArrayList<Language> langList = null;
		try {
			langList = detector.getProbabilities();
		} catch (LangDetectException e) {
			return true;
		}
		
		double englishProbability = 0;
		for (Language lang : langList){
			if (lang.lang.compareTo("en") == 0){
				englishProbability = lang.prob;
			}
		}
		if (englishProbability < PROBTHRESHOLD){
			return true;
		}
			
		return false;
	}
	
	public String getPreprocessedText(String text){
		String preprocessedText = "";
		
		String temp = text.toLowerCase();
		
		StringTokenizer tk1 = new StringTokenizer(temp);
		while (tk1.hasMoreTokens()){
			String word = tk1.nextToken();
			
			if(word.startsWith("http:")){
				temp = temp.replace(word, "");
			}
		}
		
		for (int i=0; i<temp.length(); i++){
			char k = temp.charAt(i);
			if(k != '@' && k != '-')
				if (Character.getNumericValue(k) == -1){
					temp = temp.substring(0, i) + " " + temp.substring(i+1, temp.length());
				}
		}
		
		StringTokenizer tk2 = new StringTokenizer(temp);
		
		while (tk2.hasMoreTokens()){
			String word = tk2.nextToken();
			
			if (word.startsWith("-")){
				word = word.substring(1, word.length());
			}
			
			if (word.endsWith("-")){
				word = word.substring(0,word.length()-1);
			}
			
			if (word.startsWith("@")){
				continue;
			}
			
			if(stopWords.contains(word)){
				continue;
			}
			
			if(word.compareTo("-") == 0){
				continue;
			}
			
			if(word.compareTo("@") == 0){
				continue;
			}
		
			preprocessedText += word + " ";

		}
		
		return preprocessedText;
	}
	
	private void getFilteringWords(){
		BufferedReader br = null;

		/*
		try {
			br = new BufferedReader(new FileReader("celebrities.txt"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			System.exit(-1);	
		}
		try {
			while(br.ready()){
				StringTokenizer tk = new StringTokenizer(br.readLine());
				while(tk.hasMoreTokens()){
					String next = tk.nextToken().toLowerCase();
					filteringWords.add(next);
				}
			}
			br.close();
		}catch (IOException e) {
			e.printStackTrace();
			System.exit(-1);
		}
		*/
		
		try {
			br = new BufferedReader(new FileReader("badwords.txt"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			System.exit(-1);	
		}
		try {
			while(br.ready()){
				StringTokenizer tk = new StringTokenizer(br.readLine(), ";");
				while(tk.hasMoreTokens()){
					String next = tk.nextToken();
					filteringWords.add(next);
				}
			}
			br.close();
		}catch (IOException e) {
			e.printStackTrace();
			System.exit(-1);
		}
		
		/*
		try {
			br = new BufferedReader(new FileReader("funwords.txt"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			System.exit(-1);	
		}
		try {
			while(br.ready()){
				StringTokenizer tk = new StringTokenizer(br.readLine());
				while(tk.hasMoreTokens()){
					String next = tk.nextToken();
					filteringFunWords.add(next);
				}
			}
			br.close();
		}catch (IOException e) {
			e.printStackTrace();
			System.exit(-1);
		}
		*/
	}

	private void getStopWords(){
		BufferedReader br = null;

		try {
			br = new BufferedReader(new FileReader("stopwords.txt"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			System.exit(-1);	
		}
		try {
			while(br.ready()){
				StringTokenizer tk = new StringTokenizer(br.readLine());
				while(tk.hasMoreTokens()){
					String next = tk.nextToken().toLowerCase();
					stopWords.add(next);
				}
			}
			br.close();
		}catch (IOException e) {
			e.printStackTrace();
			System.exit(-1);
		}
	}

	
}
